# !pip install pyhmy --upgrade
import json
import pandas as pd
import os
from os import path
import shutil
import re
import plotly.express as px
import plotly.graph_objects as go
import numpy as np
import gzip
from IPython.core.display import display, HTML
def read_data(node):
log_dir = '/home/ubuntu/jupyter/logs/mainnet/{}'.format(node)
files = os.listdir(log_dir)
data = []
for file in files:
if "05-06" in file:
with gzip.open(path.join(log_dir, file)) as f:
for line in f.readlines():
if '[metrics][p2p]'.encode('utf-8') in line:
data.append(json.loads(line))
return data
def data_processing(data, start_time, end_time):
df = pd.DataFrame(data, columns = ['ip', 'TotalIn', 'RateIn', 'time', 'message', 'TotalOut', 'RateOut'])
df['time'] = pd.to_datetime(df['time'], format = '%Y-%m-%dT%H:%M:%S.%f')
print(f"available time slot: from {df.iloc[0]['time']} to {df.iloc[-1]['time']}")
df = df[(df['time'] > start_time) & (df['time'] < end_time)]
df.sort_values(by=['time'], ascending=True, inplace = True)
return df
def draw_graph(df, node):
html_path = "https://harmony-one.github.io/harmony-log-analysis/graphs/p2p/{}.html".format(node)
df_in = df[df['message'] == '[metrics][p2p] traffic in in bytes']
trace1 = go.Scatter(
x = df_in['time'],
y = df_in['TotalIn'],
mode = 'lines+markers',
line = dict(color = '#00AEE9',
width = 1),
name = 'Received-in Bytes'
)
df_out = df[df['message'] == '[metrics][p2p] traffic out in bytes']
trace2 = go.Scatter(
x= df_out["time"],
y= df_out["TotalOut"],
mode='lines+markers',
line = dict(color='#FFA07A',
width=1),
name = "Sent-out Bytes",
yaxis = 'y2'
)
data = [trace1, trace2]
layout = go.Layout(
title = 'Bytes vs Time',
xaxis_title="utc_time",
yaxis=dict(
title='Received-in Bytes'
),
yaxis2=dict(
title='Sent-out Bytes',
overlaying='y',
side='right'
),
legend_orientation="h",
legend=dict(x=0, y=-0.1)
)
fig = go.Figure(data=data, layout=layout)
df_reset = df[df['message'] == '[metrics][p2p] Reset after 1 consensus cycle']
for k in df_reset['time']:
fig.add_shape(type="line", x0=k, y0=0,x1=k,y1=1,
line=dict(
width=1,
dash="dot",
))
fig.update_shapes(dict(xref='x', yref='paper'))
fig.update_layout(legend_orientation="h", legend=dict(x=0, y=-0.25))
fig.show(renderer="svg",width=800, height=500)
fig.write_html(html_dir + node + '.html')
print("HTML saved in ")
display(HTML("<a href='" + html_path + "' target='_blank'>" + html_path + "</a>"))
def analysis(node, start_time, end_time):
data = read_data(node)
df = data_processing(data, start_time, end_time)
draw_graph(df,node)
def get_df(start, end):
return df.iloc[start:end,]
html_dir = "/home/ubuntu/jupyter/harmony-log-analysis/docs/graphs/p2p/"
if not os.path.exists(html_dir):
os.makedirs(html_dir)
start_time = '2020-05-06 00:00:46'
end_time = '2020-05-06 00:10:46'
node = '34.210.74.9'
data = read_data(node)
df = data_processing(data, start_time, end_time)
get_df(10,20)
draw_graph(df,node)
node = '34.242.87.85'
analysis(node, start_time, end_time)
node = '34.244.166.68'
analysis(node, start_time, end_time)
node = '34.254.64.112'
analysis(node, start_time, end_time)
node = '34.240.243.212'
analysis(node, start_time, end_time)
node = '18.202.231.246'
analysis(node, start_time, end_time)
node = '34.244.240.175'
analysis(node, start_time, end_time)